package main

import (
    "crawler/distribute/config"
    "crawler/distribute/persist/client"
    "crawler/distribute/rpcsupport"
    processor "crawler/distribute/worker/client"
    "crawler/engine"
    "crawler/scheduler"
    "crawler/zhenai"
    "flag"
    "log"
    "net/rpc"
    "strings"
)

var processorHosts = flag.String("processorhosts", "", "processorhosts")
var itemsaverhost = flag.String("itemsaverhost", "", "itemsaverhost")

func main() {
    flag.Parse()
    itemSaver, err := client.CreateItemSaver(*itemsaverhost)
    if err != nil {
        log.Fatalf("connect to itemsaver err %s", err.Error())
    }
    
    hosts := strings.Split(*processorHosts, ",")
    log.Printf("going to connect processor hosts:%+v", hosts)
    p, err := processor.CreateProcessor(createClientPool(hosts...))
    if err != nil {
        log.Fatalf("connect to processor err %s", err.Error())
    }
    
    e := engine.ConcurrentEngine{
        Scheduler:        new(scheduler.QueuedScheduler),
        WorkCount:        100,
        ItemChan:         itemSaver,
        RequestProcessor: p,
    }
    e.Run(engine.Request{
        Url:    "http://www.zhenai.com/zhenghun",
        Parser: engine.NewFuncParser(config.ZhenAiParseCityList, zhenai.ParseCityList),
    })
}

func createClientPool(hosts ...string) chan *rpc.Client {
    var clients []*rpc.Client
    clientChan := make(chan *rpc.Client)
    for _, h := range hosts {
        client, err := rpcsupport.CreateJsonRpcClient(h)
        if err != nil {
            log.Printf("Connect to %s err:%s", h, err.Error())
            continue
        } else {
            clients = append(clients, client)
        }
    }
    
    go func() {
        for {
            for _, c := range clients {
                clientChan <- c
            }
        }
    }()
    
    return clientChan
}
